from urllib import request
import json
import pymysql
import time
from multiprocessing import Process, Lock, Queue
import _thread
import os
import sys
import traceback

# 获取的block信息放在该缓存中，以备解析
json_buf = []
# 当前最快的解析block number
block_index = 1


# 用于解析缓存中的数据，并写入数据库，此为多进程运行
def BlockPraseProcess(queue_block_buf, queue_finish):
    database = pymysql.connect("scanner_mysql","root","123456","scan" )
    cursor = database.cursor()
    while True:
        json_obj = queue_block_buf.get()
        # 计算时间
        time_stamp = 0
        for item in json_obj["extrinsics"]:
            if item["method"]["pallet"] == "timestamp" and item["method"]["method"] == "set":
                time_stamp = int(item["args"]["now"])
        try:
            # 插入block_info数据库
            block_latest = {
                "number": json_obj["number"],
                "hash": json_obj["hash"],
                "time": time_stamp
            }
            block_sql = """
                INSERT INTO BlockInfo(time,
                    number,
                    hash,
                    parentHash,
                    stateRoot,
                    extrinsicsRoot,
                    authorId,
                    finalized)
                VALUES (
                    %d, '%s','%s','%s','%s','%s','%s',%s
                );
            """%(time_stamp,
            json_obj["number"],
            json_obj["hash"],
            json_obj["parentHash"],
            json_obj["stateRoot"],
            json_obj["extrinsicsRoot"],
            json_obj["authorId"],
            json_obj["finalized"])
            # print(block_sql)
            cursor.execute(block_sql)
            cursor.execute("select last_insert_id();")
            # 插入Extrinsics数据
            results = cursor.fetchall()
            block_id = None
            for row in results:
                block_id = row[0]
            block_latest["extrinsics_count"] = len(json_obj['extrinsics'])
            for item in json_obj['extrinsics']:
                # signer有时候在["signature"]["signer"]["id"]里，有时候在["signature"]["signer"]
                signer_id = None
                if item["signature"] != None:
                    if not isinstance(item["signature"]["signer"], str) and "id" in item["signature"]["signer"]:
                        signer_id = item["signature"]["signer"]["id"]
                    else:
                        signer_id = item["signature"]["signer"]
                
                extrinsics_sql = """
                    insert into ExtrinsicsInfo(
                        block_id,
                        time,
                        pallet,
                        method,
                        signature,
                        signer,
                        nonce,
                        args,
                        hash,
                        partialFee,
                        success,
                        paysFee
                    ) values(
                        %d, %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %s, %s
                    );
                """%(block_id, 
                time_stamp,
                item["method"]["pallet"], 
                item["method"]["method"], 
                item["signature"]["signature"] if item["signature"] != None else None, 
                signer_id, 
                item["nonce"], 
                json.dumps(item["args"]), 
                item["hash"], 
                item["info"]["partialFee"] if "partialFee" in item["info"] else "0",
                item["success"],
                item["paysFee"]
                )
                cursor.execute(extrinsics_sql)
                
                # 开始获取transaction的id
                cursor.execute("select last_insert_id();")
                results = cursor.fetchall()
                extrinsics_id = None
                for row in results:
                    extrinsics_id = row[0]
                # ===================================
                # 开始解析transaction
                transaction_count = 0
                if item["method"]["pallet"] == "balances" and (item["method"]["method"] == "transferKeepAlive" or item["method"]["method"] == "transfer"):
                    #print(item)
                    #print("=======================")
                    transaction_count += 1
                    signer_id = None
                    if item["signature"] != None:
                        if not isinstance(item["signature"]["signer"], str) and "id" in item["signature"]["signer"]:
                            signer_id = item["signature"]["signer"]["id"]
                        else:
                            signer_id = item["signature"]["signer"]
                    dest = None
                    if not isinstance(item["args"]["dest"], str) and "id" in item["args"]["dest"]:
                        dest = item["args"]["dest"]["id"]
                    else:
                        dest = item["args"]["dest"]

                    transaction_sql = """insert into TransactionInfo(src, dest, amount, extrinsics_id, time, block_id) values('%s', '%s', '%s', %d, %d, %d)"""%(
                        signer_id,
                        dest if isinstance(dest, str) else json.dumps(dest),
                        item["args"]["value"],
                        extrinsics_id,
                        time_stamp,
                        block_id
                    )
                    cursor.execute(transaction_sql)
                    # ===================================
                    # 开始将交易存入最新交易中
                    trasaction_latest = """
                    insert into  
                    TransactionInfoLatest(`time`, `src`, `dest`, `amount`, `hash`)
                    values(%d, '%s', '%s', '%s', '%s')
                    """%(time_stamp, signer_id, dest if isinstance(dest, str) else json.dumps(dest), item["args"]["value"], item["hash"])
                    cursor.execute(trasaction_latest)
                block_latest["transaction_count"] = transaction_count
                # ===================================
                # 开始解析event
                for event in item["events"]:
                    event_sql = """
                    insert into EventInfo(extrinsics_id, time, pallet, method, data, block_id) values(
                        %d, %d, '%s', '%s', '%s', %d
                    )
                    """%(extrinsics_id, time_stamp, event["method"]["pallet"], event["method"]["method"], json.dumps(event["data"]), block_id)
                    cursor.execute(event_sql)
            # ===================================
            # 开始将区块存入最新区块中
            block_sql_latest = """
            insert into  
            BlockInfoLatest(`time`, `number`, `hash`, `extrinsics_count`, `transaction_count`)
            values(%d, '%s', '%s', %d, %d)
            """%(block_latest["time"], block_latest["number"], block_latest["hash"], block_latest["extrinsics_count"], block_latest["transaction_count"])
            cursor.execute(block_sql_latest)
            # ===================================
            # 将最新区块和最新交易中老的删除
            sql_tmp = """
            delete from
            TransactionInfoLatest
            where id not in (select t.id from (SELECT id FROM TransactionInfoLatest order by time desc limit 0,100) as t);
            """
            cursor.execute(sql_tmp)

            sql_tmp = """
            delete from
            BlockInfoLatest
            where id not in (select t.id from (SELECT id FROM BlockInfoLatest order by time desc limit 0,100) as t);
            """
            cursor.execute(sql_tmp)
            # 提交到数据库执行
            database.commit()
            queue_finish.put(json_obj["number"])
        except KeyboardInterrupt as e:
            # 如果发生错误则回滚
            print("插入数据库发生错误")
            database.rollback()
            exit()
        except pymysql.err.IntegrityError as e:
            if str(e).startswith("(1062") and str(e).endswith("""for key 'BlockInfo.number'")"""):
                print("该block已经解析过，跳过:%s"%json_obj["number"])
                database.rollback()
                queue_finish.put(json_obj["number"])
                continue
            else:
                #该处表示没有处理的异常
                print("===============================================")
                print(json.dumps(json_obj))
                print('str(Exception):\t', str(Exception))
                print('str(e):\t\t', str(e))
                print('repr(e):\t', repr(e))
                # Get information about the exception that is currently being handled  
                exc_type, exc_value, exc_traceback = sys.exc_info() 
                print('e.message:\t', exc_value)
                print("Note, object e and exc of Class %s is %s the same." % 
                        (type(exc_value), ('not', '')[exc_value is e]))
                print('traceback.print_exc(): ', traceback.print_exc())
                print('traceback.format_exc():\n%s' % traceback.format_exc())
                exit()
        except Exception as e:
            #该处表示没有处理的异常
            print("===============================================")
            print(json.dumps(json_obj))
            print('str(Exception):\t', str(Exception))
            print('str(e):\t\t', str(e))
            print('repr(e):\t', repr(e))
            # Get information about the exception that is currently being handled  
            exc_type, exc_value, exc_traceback = sys.exc_info() 
            print('e.message:\t', exc_value)
            print("Note, object e and exc of Class %s is %s the same." % 
                    (type(exc_value), ('not', '')[exc_value is e]))
            print('traceback.print_exc(): ', traceback.print_exc())
            print('traceback.format_exc():\n%s' % traceback.format_exc())
            exit()
        finally:
            pass
        
# 获取block信息，并将其加入缓存
def GetBlock():
    global json_buf
    while True:
        block_idx = GetBlockIndexNow()
        while True:
            try:
                # 首先先找到最新的区块，如果小于最新区块则get，否则就等待
                while True:
                    res = request.urlopen('http://scanner_sidecar:8080/blocks/head', timeout = 10)
                    html = res.read()
                    json_obj = json.loads(html)
                    latest_block_number = int(json_obj["number"])
                    if block_index <= latest_block_number:
                        break
                    else:
                        print("当前区块为%d最新"%latest_block_number)
                        time.sleep(1)
                # 开始获取block信息
                res = request.urlopen('http://scanner_sidecar:8080/blocks/%d'%block_idx, timeout = 300)
                html = res.read()
                json_obj = json.loads(html)
                if json_obj["finalized"] == False:
                    time.sleep(1)
                    continue
                json_buf.append(json_obj)
                if block_idx%1 == 0:
                    print("获取%d的区块, 当前待解析区块为%d，%s"%(block_idx, len(json_buf), time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))))
                break
            except KeyboardInterrupt as e:
                print("手动中断",e)
                exit()
            except:
                print("未知异常，打概率是超时，如果计算机速度较慢，请提高超时时间")
                exit()
            


# 初始化上一次加载到的index
def InitMinIndex():
    global block_index
    block_index = GetBlockMinIndex()
    print("上一次采集到block: %d"%block_index)
    if block_index == 1:# 第一次的情况，保存状态文件
        file = open("./latest", "wt")
        file.write(json.dumps({"index":1}))
        file.close()

def GetBlockMinIndex():
    try:
        file = open("./latest", "r")
        content = file.read()
        json_obj = json.loads(content)
        file.close()
        return json_obj["index"]
    except:
        try:
            file = open("./latest_back", "r")
            content = file.read()
            json_obj = json.loads(content)
            file.close()
            return json_obj["index"]
        except:
            return 1

# 更新最新的block
def UpdateMinIndex(index):
    os.rename("./latest", "./latest_back")
    file = open("./latest", "wt")
    file.write(json.dumps({"index":index}))
    file.close()

# 获取当前要采集的block
def GetBlockIndexNow():
    global block_index
    block_index_tmp = block_index
    block_index += 1
    return block_index_tmp

#更新下载好等待处理的block
def UploadWaitBlock():
    pass

# 更新以处理的最小block
def UploadMinBlockThread(queue):
    start_index = GetBlockMinIndex()
    buf = []
    while True:
        index = int(queue.get())
        buf.append(index)
        if index == start_index:
            while start_index in buf:
                buf.remove(start_index)
                UpdateMinIndex(start_index)
                start_index += 1
                if start_index % 1000 == 0:
                    print("已处理序列化block:%d, 剩余%d"%(index, len(buf)))

# 启动多线程获取block
def MutiThreadGetBlock():
    thread_count = 1
    for i in range(thread_count):
        _thread.start_new_thread( GetBlock, ( ) )

# 多进程用于处理block
def MuteProcessGetBlock():
    process_count = 1
    global json_buf
    queue_block_buf = Queue()
    queue_finish = Queue()
    lock = Lock()
    for i in range(process_count):
        process = Process(target = BlockPraseProcess, args = (queue_block_buf,queue_finish))
        process.start()
    _thread.start_new_thread(UploadMinBlockThread, (queue_finish,))
    while True:
        if len(json_buf) > 0:
            json_obj = json_buf.pop(0)
            queue_block_buf.put(json_obj)
        else:
            #print("no new block")
            time.sleep(0.5)







def test():
    json_obj = json.loads("""{"number": "304750", "hash": "0x423f89df850071a72d9f2c3a0ce0a8141365301a3b7019e1482b5514fe7f7642", "parentHash": "0xb65ea1dfbe73aea46554e6a291e0a878e2a18dc05acc64dd877cfdfca7205d7d", "stateRoot": "0x32c4863d5b1a237e2065b75ddd8cd7a12fcf25a99a247e0a112a274d7746daf7", "extrinsicsRoot": "0x83c24d6a3fd6f062abef502de7efcbb7443f23fe358cf06c0eca47ff23ce74ff", "authorId": "13wY4rD88C3Xzd4brFMPkAMEMC3dSuAR2NC6PZ5BEsZ5t6rJ", "logs": [{"type": "PreRuntime", "index": "6", "value": ["0x42414245", "0x03040000002d85d10f0000000020570a95de6732f291b5582a8072e26d34bc60055bea8e593d0471222acf96156897377daa4a02b9115623be38debc2ec8549f069a3761509af8080d60248b0d51537bfa79dc66384a49a18c93c134518a7f212beb5012d331f19d45673fc303"]}, {"type": "Seal", "index": "5", "value": ["0x42414245", "0xfc04923e29b49668f77b917ef1834aa4378f31c60e7d8d30df3f8d0044b05f51cc7b5998d5ff6008dbdebc9db54ae6031e2935f8607c94524fd530154d1dce84"]}], "onInitialize": {"events": []}, "extrinsics": [{"method": {"pallet": "timestamp", "method": "set"}, "signature": null, "nonce": null, "args": {"now": "1592336142000"}, "tip": null, "hash": "0xae64254107598ea4f865c42d46e9fa54bca8c0a505606968f62c8c9f15b7d234", "info": {}, "events": [{"method": {"pallet": "system", "method": "ExtrinsicSuccess"}, "data": [{"weight": "158000000", "class": "Mandatory", "paysFee": "Yes"}]}], "success": true, "paysFee": false}, {"method": {"pallet": "finalityTracker", "method": "finalHint"}, "signature": null, "nonce": null, "args": {"hint": "304747"}, "tip": null, "hash": "0x06f7a801fd6ebce30d165a59547fffd0706040675674c202650ed9cfa5e42b88", "info": {}, "events": [{"method": {"pallet": "system", "method": "ExtrinsicSuccess"}, "data": [{"weight": "0", "class": "Mandatory", "paysFee": "Yes"}]}], "success": true, "paysFee": false}, {"method": {"pallet": "parachains", "method": "setHeads"}, "signature": null, "nonce": null, "args": {"heads": []}, "tip": null, "hash": "0xcf52705d1ade64fc0b05859ac28358c0770a217dd76b75e586ae848c56ae810d", "info": {}, "events": [{"method": {"pallet": "system", "method": "ExtrinsicSuccess"}, "data": [{"weight": "1000000000", "class": "Mandatory", "paysFee": "Yes"}]}], "success": true, "paysFee": false}, {"method": {"pallet": "balances", "method": "transfer"}, "signature": {"signature": "0x0a4ebaaea56444fa64a59a26dcb245dc90a5bc28a7601d3fe121242d245d0b2ebb39578d809861870f631c8b5754b2c5aaadc3df77418c8df2ab0ea5168b0a87", "signer": "169yznAaSLYiEjEFWpC7Ao6xJVBSV9r7cz9G688h48EcVX6L"}, "nonce": "2", "args": {"dest": "129367mjzvDwiP26DSdZxuar8i1BeR3oTPkUCgWbf6KUTzVQ", "value": "123000000000"}, "tip": "0", "hash": "0x6d44a411fe5c8abcd7c6583ec8120380a95e3fdd237c69db21c109c4f10b021f", "info": {"weight": "195000000", "class": "Normal", "partialFee": "10000000"}, "events": [{"method": {"pallet": "treasury", "method": "Deposit"}, "data": ["8000000"]}, {"method": {"pallet": "balances", "method": "Deposit"}, "data": ["13wY4rD88C3Xzd4brFMPkAMEMC3dSuAR2NC6PZ5BEsZ5t6rJ", "2000000"]}, {"method": {"pallet": "system", "method": "ExtrinsicFailed"}, "data": [{"badOrigin": null}, {"weight": "195000000", "class": "Normal", "paysFee": "Yes"}]}], "success": false, "paysFee": true}], "onFinalize": {"events": []}, "finalized": true}""")
        # 计算时间
    for item in json_obj["extrinsics"]:
        if item["method"]["pallet"] == "timestamp" and item["method"]["method"] == "set":
            time_stamp = int(item["args"]["now"])

    # 插入block_info数据库
    block_sql = """
        INSERT INTO BlockInfo(time,
            number,
            hash,
            parentHash,
            stateRoot,
            extrinsicsRoot,
            authorId,
            finalized)
        VALUES (
            %d, '%s','%s','%s','%s','%s','%s',%s
        );
    """%(time_stamp,
    json_obj["number"],
    json_obj["hash"],
    json_obj["parentHash"],
    json_obj["stateRoot"],
    json_obj["extrinsicsRoot"],
    json_obj["authorId"],
    json_obj["finalized"])
    
    block_id = 1

    for item in json_obj['extrinsics']:
        print(json.dumps(item))
        signer_id = None
        if item["signature"] != None:
            
            if not isinstance(item["signature"]["signer"], str) and "id" in item["signature"]["signer"]:
                signer_id = item["signature"]["signer"]["id"]
            else:
                signer_id = item["signature"]["signer"]
        extrinsics_sql = """
            insert into ExtrinsicsInfo(
                block_id,
                time,
                pallet,
                method,
                signature,
                signer,
                nonce,
                args,
                hash,
                partialFee,
                success,
                paysFee
            ) values(
                %d, %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %s, %s
            );
        """%(block_id, 
        time_stamp,
        item["method"]["pallet"], 
        item["method"]["method"], 
        item["signature"]["signature"] if item["signature"] != None else None, 
        signer_id, 
        item["nonce"], 
        json.dumps(item["args"]), 
        item["hash"], 
        item["info"]["partialFee"] if "partialFee" in item["info"] else "0",
        item["success"],
        item["paysFee"]
        )
        print(signer_id)
        extrinsics_id = 1

        # ===================================
        # 开始解析transaction
        if item["method"]["pallet"] == "balances" and (item["method"]["method"] == "transferKeepAlive" or item["method"]["method"] == "transfer"):
            #print(item)
            #print("=======================")
            signer_id = None
            if item["signature"] != None:
                if not isinstance(item["signature"]["signer"], str) and "id" in item["signature"]["signer"]:
                    signer_id = item["signature"]["signer"]["id"]
                else:
                    signer_id = item["signature"]["signer"]
            dest = None
            if not isinstance(item["args"]["dest"], str) and "id" in item["args"]["dest"]:
                dest = item["args"]["dest"]["id"]
            else:
                dest = item["args"]["dest"]
            transaction_sql = """insert into TransactionInfo(src, dest, amount, extrinsics_id, time) values('%s', '%s', %d, %d, %d)"""%(
                signer_id,
                dest,
                int(item["args"]["value"]),
                extrinsics_id,
                time_stamp
            )


        # ===================================
        # 开始解析event
        for event in item["events"]:
            event_sql = """
            insert into EventInfo(extrinsics_id, time, pallet, method, data) values(
                %d, %d, '%s', '%s', '%s'
            )
            """%(extrinsics_id, time_stamp, event["method"]["pallet"], event["method"]["method"], json.dumps(event["data"]))

if __name__ == '__main__':
    InitMinIndex()
    MutiThreadGetBlock()
    MuteProcessGetBlock()